//: -L+pcred+pcre

import 
	pcred.RegExp,
	tango.net.http.HttpGet,
	tango.io.device.Array,
	tango.io.device.File,
	tango.io.Stdout;

auto base_url	= `http://bibliodiversidad.com/innovaeditor/` ;
RegExp re1 ;
Array bu ;

void main(){
	bu	= new Array(1024, 1024);
	re1	= new RegExp(`a href="(.+?)">\s+(.+?)<`);
	dl(``);
	File.set(`url.txt`, bu.slice );
}

void dl(char[] _path ){
	assert(_path !is null);
	auto url	=  base_url ~ _path ;
	auto u		= new HttpGet(url) ;
	u.setTimeout(300);
	Stdout.formatln("{}", url ).flush ;
	
	auto da		= cast(char[]) u.read() ;
	assert(u.isResponseOK);
	
	if( _path.length is 0 || _path.length > 0 && _path[$-1] is '/' ) {
		auto ms	= re1.findAll(da);
		if(  ms !is null) foreach(int i, ma; ms) if( i ) {
			auto __path	= _path ~ ma[1] ;
			
			if( __path[$-1] is '/' ) {
				dl(__path);
			} else {
				bu(base_url)(__path)("\n") ;
			}
		}
		return ;
	}
}